{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = 'https://f000.backblazeb2.com/file/malay-dataset/'\n",
    "\n",
    "s = \"\"\"\n",
    "1. translation/malay-english/translated-0.json\n",
    "2. translation/malay-english/translated-100000.json\n",
    "3. translation/malay-english/translated-200000.json\n",
    "4. translation/malay-english/translated-300000.json\n",
    "5. translation/malay-english/translated-400000.json\n",
    "6. translation/malay-english/translated-500000.json\n",
    "7. translation/malay-english/translated-600000.json\n",
    "8. translation/malay-english/translated-700000.json\n",
    "9. translation/malay-english/translated-800000.json\n",
    "10. translation/malay-english/translated-900000.json\n",
    "11. translation/malay-english/translated-1000000.json\n",
    "12. translation/malay-english/translated-1100000.json\n",
    "13. translation/malay-english/translated-1200000.json\n",
    "14. translation/malay-english/translated-1300000.json\n",
    "15. translation/malay-english/translated-1400000.json\n",
    "16. translation/malay-english/translated-1500000.json\n",
    "17. translation/malay-english/translated-1600000.json\n",
    "18. translation/malay-english/translated-1700000.json\n",
    "19. translation/malay-english/translated-1800000.json\n",
    "20. translation/malay-english/translated-1900000.json\n",
    "21. translation/malay-english/translated-2000000.json\n",
    "22. translation/malay-english/translated-2100000.json\n",
    "23. translation/malay-english/translated-2200000.json\n",
    "24. translation/malay-english/translated-2300000.json\n",
    "25. translation/malay-english/translated-2400000.json\n",
    "26. translation/malay-english/translated-2500000.json\n",
    "27. translation/malay-english/translated-2600000.json\n",
    "28. translation/malay-english/translated-2700000.json\n",
    "29. translation/malay-english/translated-2800000.json\n",
    "30. translation/malay-english/translated-2900000.json\n",
    "31. translation/malay-english/translated-3000000.json\n",
    "32. translation/malay-english/translated-3100000.json\n",
    "33. translation/malay-english/translated-3200000.json\n",
    "34. translation/malay-english/translated-3300000.json\n",
    "35. translation/malay-english/translated-3400000.json\n",
    "36. translation/malay-english/translated-3500000.json\n",
    "\"\"\"\n",
    "s = list(filter(None, s.split('\\n')))\n",
    "s = [i.split('. ')[1] for i in s]\n",
    "s = [f'{prefix}{i}' for i in s]\n",
    "\n",
    "for i in s:\n",
    "    os.system(f'wget {i}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = 'https://f000.backblazeb2.com/file/malay-dataset/'\n",
    "\n",
    "s = \"\"\"\n",
    "1. translation/english-news/dataset-0.json.translate\n",
    "2. translation/english-news/dataset-100000.json.translate\n",
    "3. translation/english-news/dataset-200000.json.translate\n",
    "4. translation/english-news/dataset-300000.json.translate\n",
    "5. translation/english-news/dataset-400000.json.translate\n",
    "6. translation/english-news/dataset-500000.json.translate\n",
    "7. translation/english-news/dataset-600000.json.translate\n",
    "8. translation/english-news/dataset-700000.json.translate\n",
    "9. translation/english-news/dataset-800000.json.translate\n",
    "10. translation/english-news/texts.json.translate\n",
    "11. translation/english-news/dataset-900000.json.translate\n",
    "12. translation/english-news/dataset-1000000.json.translate\n",
    "13. translation/english-news/dataset-1100000.json.translate\n",
    "14. translation/english-news/dataset-1200000.json.translate\n",
    "15. translation/english-news/dataset-1300000.json.translate\n",
    "\"\"\"\n",
    "\n",
    "s = list(filter(None, s.split('\\n')))\n",
    "s = [i.split('. ')[1] for i in s]\n",
    "s = [f'{prefix}{i}' for i in s]\n",
    "\n",
    "for i in s:\n",
    "    os.system(f'wget {i}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.system('wget https://f000.backblazeb2.com/file/malay-dataset/translation/parliament/translated-trainset-parliament.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "\n",
    "left, right = [], []\n",
    "files = glob('alignment/translated*.json')\n",
    "for f in files:\n",
    "    print(f)\n",
    "    with open(f) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    for i in data:\n",
    "        left.append(i[1])\n",
    "        right.append(i[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = glob('alignment/dataset-*.json.translate')\n",
    "for f in files:\n",
    "    print(f)\n",
    "    with open(f) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    for i in data:\n",
    "        left.append(i[0])\n",
    "        right.append(i[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('source', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(left))\n",
    "    \n",
    "with open('target', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(right))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
